package util.HtmlUnit;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.http.client.params.CookiePolicy;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.ProxyConfig;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.util.Cookie;

public class SpiderContentTxt {


	public String SpiderPageContent(String site){
//System.out.println(site);
		String bodyString = null;
		HtmlPage spiderPage = null;
		WebClient webClient = new WebClient(BrowserVersion.INTERNET_EXPLORER_8);
		// webClient.getCookieManager().setCookiesEnabled(false);
		System.setProperty("apache.commons.httpclient.cookiespec", CookiePolicy.BROWSER_COMPATIBILITY);
		// webClient关闭javascript、设置最长连接时间15秒
		webClient.setJavaScriptEnabled(false);
		webClient.setCssEnabled(false);
		webClient.setTimeout(15000);

		if (proxyString.length() > 3) {
			Pattern p = Pattern.compile("(.*)(:)(.*)");
			Matcher m = p.matcher(proxyString);
			if (m.find()) {
				ProxyConfig prc = new ProxyConfig(m.group(1), new Integer(m.group(3)).intValue());
				webClient.setProxyConfig(prc);
			}
		}
		if (cookiesString.length() > 3) {
			Pattern p = Pattern.compile("(<<<(.*)>>>,)(<<<(.*)>>>,)(<<<(.*)>>>)");
			Matcher m = p.matcher(cookiesString);
			if (m.find()) {
				webClient.getCookieManager().addCookie(new Cookie(m.group(2), m.group(4), m.group(6)));
			}
		}

		ArrayList<String> siteList = new ArrayList<String>();
		// sougou是直接封，如果被封下一句就会出异常，跳到catch块
		// System.out.println("网页抓取开始");
		siteList.add(0, site);
		try {
			//System.out.println(siteList.get(0));
			spiderPage = webClient.getPage(siteList.get(0));

			bodyString = spiderPage.asXml();
			// System.out.println(PagingRex);
			if (PagingRex.length() > 2) {
				Pattern p = Pattern.compile(PagingRex);
				Matcher m = p.matcher(bodyString);
				while (m.find()) {
					bodyString = ((HtmlPage) webClient.getPage(PagingRexHead + m.group(2).replaceAll("&amp;", "&"))).getBody().asXml();
				}
			}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			//e.printStackTrace();
			bodyString = "";
		} catch (IOException e) {
			// TODO Auto-generated catch block
			//e.printStackTrace();
			bodyString = "";
		} finally {

			if (webClient != null) {
				webClient.closeAllWindows();
				webClient = null;
			}
		}
//System.out.println(bodyString);
		return bodyString;
	}

	String cookiesString = "";

	public void setCookiesString(String cookiesString) {
		this.cookiesString = cookiesString;
	}

	public void setProxyString(String proxyString) {
		this.proxyString = proxyString;
	}

	String proxyString = "";

	String PagingRex = "";

	public void setPagingRex(String pagingRex) {
		this.PagingRex = pagingRex;
		// System.out.println(PagingRex);
	}

	public void setPagingRexHead(String pagingRexHead) {
		this.PagingRexHead = pagingRexHead;
	}

	String PagingRexHead = "";
}
